{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING (theano.configdefaults): g++ not available, if using conda: `conda install m2w64-toolchain`\n",
      "C:\\Users\\yinahe\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\theano\\configdefaults.py:560: UserWarning: DeprecationWarning: there is no c++ compiler.This is deprecated and with Theano 0.11 a c++ compiler will be mandatory\n",
      "  warnings.warn(\"DeprecationWarning: there is no c++ compiler.\"\n",
      "WARNING (theano.configdefaults): g++ not detected ! Theano will be unable to execute optimized C-implementations (for both CPU and GPU) and will default to Python implementations. Performance will be severely degraded. To remove this warning, set Theano flags cxx to an empty string.\n",
      "WARNING (theano.tensor.blas): Using NumPy C-API based implementation for BLAS functions.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trying to run under a GPU.  If this is not desired, then modify network3.py\n",
      "to set the GPU flag to False.\n"
     ]
    }
   ],
   "source": [
    "import network3\n",
    "from network3 import Network\n",
    "from network3 import ConvPoolLayer, FullyConnectedLayer, SoftmaxLayer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_data, validation_data, test_data = network3.load_data_shared()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mini_batch_size = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "net = Network([\n",
    "        FullyConnectedLayer(n_in=784, n_out=100),\n",
    "        SoftmaxLayer(n_in=100, n_out=10)], mini_batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training mini-batch number 0\n",
      "Training mini-batch number 1000\n",
      "Training mini-batch number 2000\n",
      "Training mini-batch number 3000\n",
      "Training mini-batch number 4000\n",
      "Epoch 0: validation accuracy 92.75%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 91.95%\n",
      "Training mini-batch number 5000\n",
      "Training mini-batch number 6000\n",
      "Training mini-batch number 7000\n",
      "Training mini-batch number 8000\n",
      "Training mini-batch number 9000\n",
      "Epoch 1: validation accuracy 94.73%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 94.18%\n",
      "Training mini-batch number 10000\n",
      "Training mini-batch number 11000\n",
      "Training mini-batch number 12000\n",
      "Training mini-batch number 13000\n",
      "Training mini-batch number 14000\n",
      "Epoch 2: validation accuracy 95.82%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 95.24%\n",
      "Training mini-batch number 15000\n",
      "Training mini-batch number 16000\n",
      "Training mini-batch number 17000\n",
      "Training mini-batch number 18000\n",
      "Training mini-batch number 19000\n",
      "Epoch 3: validation accuracy 96.45%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 95.88%\n",
      "Training mini-batch number 20000\n",
      "Training mini-batch number 21000\n",
      "Training mini-batch number 22000\n",
      "Training mini-batch number 23000\n",
      "Training mini-batch number 24000\n",
      "Epoch 4: validation accuracy 96.66%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.38%\n",
      "Training mini-batch number 25000\n",
      "Training mini-batch number 26000\n",
      "Training mini-batch number 27000\n",
      "Training mini-batch number 28000\n",
      "Training mini-batch number 29000\n",
      "Epoch 5: validation accuracy 96.87%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.62%\n",
      "Training mini-batch number 30000\n",
      "Training mini-batch number 31000\n",
      "Training mini-batch number 32000\n",
      "Training mini-batch number 33000\n",
      "Training mini-batch number 34000\n",
      "Epoch 6: validation accuracy 97.04%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.76%\n",
      "Training mini-batch number 35000\n",
      "Training mini-batch number 36000\n",
      "Training mini-batch number 37000\n",
      "Training mini-batch number 38000\n",
      "Training mini-batch number 39000\n",
      "Epoch 7: validation accuracy 97.12%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 96.98%\n",
      "Training mini-batch number 40000\n",
      "Training mini-batch number 41000\n",
      "Training mini-batch number 42000\n",
      "Training mini-batch number 43000\n",
      "Training mini-batch number 44000\n",
      "Epoch 8: validation accuracy 97.25%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.15%\n",
      "Training mini-batch number 45000\n",
      "Training mini-batch number 46000\n",
      "Training mini-batch number 47000\n",
      "Training mini-batch number 48000\n",
      "Training mini-batch number 49000\n",
      "Epoch 9: validation accuracy 97.32%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.26%\n",
      "Training mini-batch number 50000\n",
      "Training mini-batch number 51000\n",
      "Training mini-batch number 52000\n",
      "Training mini-batch number 53000\n",
      "Training mini-batch number 54000\n",
      "Epoch 10: validation accuracy 97.40%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.32%\n",
      "Training mini-batch number 55000\n",
      "Training mini-batch number 56000\n",
      "Training mini-batch number 57000\n",
      "Training mini-batch number 58000\n",
      "Training mini-batch number 59000\n",
      "Epoch 11: validation accuracy 97.42%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.39%\n",
      "Training mini-batch number 60000\n",
      "Training mini-batch number 61000\n",
      "Training mini-batch number 62000\n",
      "Training mini-batch number 63000\n",
      "Training mini-batch number 64000\n",
      "Epoch 12: validation accuracy 97.42%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.45%\n",
      "Training mini-batch number 65000\n",
      "Training mini-batch number 66000\n",
      "Training mini-batch number 67000\n",
      "Training mini-batch number 68000\n",
      "Training mini-batch number 69000\n",
      "Epoch 13: validation accuracy 97.49%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.47%\n",
      "Training mini-batch number 70000\n",
      "Training mini-batch number 71000\n",
      "Training mini-batch number 72000\n",
      "Training mini-batch number 73000\n",
      "Training mini-batch number 74000\n",
      "Epoch 14: validation accuracy 97.51%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.51%\n",
      "Training mini-batch number 75000\n",
      "Training mini-batch number 76000\n",
      "Training mini-batch number 77000\n",
      "Training mini-batch number 78000\n",
      "Training mini-batch number 79000\n",
      "Epoch 15: validation accuracy 97.56%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.56%\n",
      "Training mini-batch number 80000\n",
      "Training mini-batch number 81000\n",
      "Training mini-batch number 82000\n",
      "Training mini-batch number 83000\n",
      "Training mini-batch number 84000\n",
      "Epoch 16: validation accuracy 97.58%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.63%\n",
      "Training mini-batch number 85000\n",
      "Training mini-batch number 86000\n",
      "Training mini-batch number 87000\n",
      "Training mini-batch number 88000\n",
      "Training mini-batch number 89000\n",
      "Epoch 17: validation accuracy 97.61%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.66%\n",
      "Training mini-batch number 90000\n",
      "Training mini-batch number 91000\n",
      "Training mini-batch number 92000\n",
      "Training mini-batch number 93000\n",
      "Training mini-batch number 94000\n",
      "Epoch 18: validation accuracy 97.63%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.64%\n",
      "Training mini-batch number 95000\n",
      "Training mini-batch number 96000\n",
      "Training mini-batch number 97000\n",
      "Training mini-batch number 98000\n",
      "Training mini-batch number 99000\n",
      "Epoch 19: validation accuracy 97.65%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.65%\n",
      "Training mini-batch number 100000\n",
      "Training mini-batch number 101000\n",
      "Training mini-batch number 102000\n",
      "Training mini-batch number 103000\n",
      "Training mini-batch number 104000\n",
      "Epoch 20: validation accuracy 97.68%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.65%\n",
      "Training mini-batch number 105000\n",
      "Training mini-batch number 106000\n",
      "Training mini-batch number 107000\n",
      "Training mini-batch number 108000\n",
      "Training mini-batch number 109000\n",
      "Epoch 21: validation accuracy 97.67%\n",
      "Training mini-batch number 110000\n",
      "Training mini-batch number 111000\n",
      "Training mini-batch number 112000\n",
      "Training mini-batch number 113000\n",
      "Training mini-batch number 114000\n",
      "Epoch 22: validation accuracy 97.68%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.66%\n",
      "Training mini-batch number 115000\n",
      "Training mini-batch number 116000\n",
      "Training mini-batch number 117000\n",
      "Training mini-batch number 118000\n",
      "Training mini-batch number 119000\n",
      "Epoch 23: validation accuracy 97.70%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.67%\n",
      "Training mini-batch number 120000\n",
      "Training mini-batch number 121000\n",
      "Training mini-batch number 122000\n",
      "Training mini-batch number 123000\n",
      "Training mini-batch number 124000\n",
      "Epoch 24: validation accuracy 97.77%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.66%\n",
      "Training mini-batch number 125000\n",
      "Training mini-batch number 126000\n",
      "Training mini-batch number 127000\n",
      "Training mini-batch number 128000\n",
      "Training mini-batch number 129000\n",
      "Epoch 25: validation accuracy 97.76%\n",
      "Training mini-batch number 130000\n",
      "Training mini-batch number 131000\n",
      "Training mini-batch number 132000\n",
      "Training mini-batch number 133000\n",
      "Training mini-batch number 134000\n",
      "Epoch 26: validation accuracy 97.75%\n",
      "Training mini-batch number 135000\n",
      "Training mini-batch number 136000\n",
      "Training mini-batch number 137000\n",
      "Training mini-batch number 138000\n",
      "Training mini-batch number 139000\n",
      "Epoch 27: validation accuracy 97.76%\n",
      "Training mini-batch number 140000\n",
      "Training mini-batch number 141000\n",
      "Training mini-batch number 142000\n",
      "Training mini-batch number 143000\n",
      "Training mini-batch number 144000\n",
      "Epoch 28: validation accuracy 97.76%\n",
      "Training mini-batch number 145000\n",
      "Training mini-batch number 146000\n",
      "Training mini-batch number 147000\n",
      "Training mini-batch number 148000\n",
      "Training mini-batch number 149000\n",
      "Epoch 29: validation accuracy 97.76%\n",
      "Training mini-batch number 150000\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training mini-batch number 151000\n",
      "Training mini-batch number 152000\n",
      "Training mini-batch number 153000\n",
      "Training mini-batch number 154000\n",
      "Epoch 30: validation accuracy 97.76%\n",
      "Training mini-batch number 155000\n",
      "Training mini-batch number 156000\n",
      "Training mini-batch number 157000\n",
      "Training mini-batch number 158000\n",
      "Training mini-batch number 159000\n",
      "Epoch 31: validation accuracy 97.73%\n",
      "Training mini-batch number 160000\n",
      "Training mini-batch number 161000\n",
      "Training mini-batch number 162000\n",
      "Training mini-batch number 163000\n",
      "Training mini-batch number 164000\n",
      "Epoch 32: validation accuracy 97.72%\n",
      "Training mini-batch number 165000\n",
      "Training mini-batch number 166000\n",
      "Training mini-batch number 167000\n",
      "Training mini-batch number 168000\n",
      "Training mini-batch number 169000\n",
      "Epoch 33: validation accuracy 97.73%\n",
      "Training mini-batch number 170000\n",
      "Training mini-batch number 171000\n",
      "Training mini-batch number 172000\n",
      "Training mini-batch number 173000\n",
      "Training mini-batch number 174000\n",
      "Epoch 34: validation accuracy 97.71%\n",
      "Training mini-batch number 175000\n",
      "Training mini-batch number 176000\n",
      "Training mini-batch number 177000\n",
      "Training mini-batch number 178000\n",
      "Training mini-batch number 179000\n",
      "Epoch 35: validation accuracy 97.70%\n",
      "Training mini-batch number 180000\n",
      "Training mini-batch number 181000\n",
      "Training mini-batch number 182000\n",
      "Training mini-batch number 183000\n",
      "Training mini-batch number 184000\n",
      "Epoch 36: validation accuracy 97.68%\n",
      "Training mini-batch number 185000\n",
      "Training mini-batch number 186000\n",
      "Training mini-batch number 187000\n",
      "Training mini-batch number 188000\n",
      "Training mini-batch number 189000\n",
      "Epoch 37: validation accuracy 97.67%\n",
      "Training mini-batch number 190000\n",
      "Training mini-batch number 191000\n",
      "Training mini-batch number 192000\n",
      "Training mini-batch number 193000\n",
      "Training mini-batch number 194000\n",
      "Epoch 38: validation accuracy 97.66%\n",
      "Training mini-batch number 195000\n",
      "Training mini-batch number 196000\n",
      "Training mini-batch number 197000\n",
      "Training mini-batch number 198000\n",
      "Training mini-batch number 199000\n",
      "Epoch 39: validation accuracy 97.64%\n",
      "Training mini-batch number 200000\n",
      "Training mini-batch number 201000\n",
      "Training mini-batch number 202000\n",
      "Training mini-batch number 203000\n",
      "Training mini-batch number 204000\n",
      "Epoch 40: validation accuracy 97.65%\n",
      "Training mini-batch number 205000\n",
      "Training mini-batch number 206000\n",
      "Training mini-batch number 207000\n",
      "Training mini-batch number 208000\n",
      "Training mini-batch number 209000\n",
      "Epoch 41: validation accuracy 97.67%\n",
      "Training mini-batch number 210000\n",
      "Training mini-batch number 211000\n",
      "Training mini-batch number 212000\n",
      "Training mini-batch number 213000\n",
      "Training mini-batch number 214000\n",
      "Epoch 42: validation accuracy 97.69%\n",
      "Training mini-batch number 215000\n",
      "Training mini-batch number 216000\n",
      "Training mini-batch number 217000\n",
      "Training mini-batch number 218000\n",
      "Training mini-batch number 219000\n",
      "Epoch 43: validation accuracy 97.69%\n",
      "Training mini-batch number 220000\n",
      "Training mini-batch number 221000\n",
      "Training mini-batch number 222000\n",
      "Training mini-batch number 223000\n",
      "Training mini-batch number 224000\n",
      "Epoch 44: validation accuracy 97.69%\n",
      "Training mini-batch number 225000\n",
      "Training mini-batch number 226000\n",
      "Training mini-batch number 227000\n",
      "Training mini-batch number 228000\n",
      "Training mini-batch number 229000\n",
      "Epoch 45: validation accuracy 97.68%\n",
      "Training mini-batch number 230000\n",
      "Training mini-batch number 231000\n",
      "Training mini-batch number 232000\n",
      "Training mini-batch number 233000\n",
      "Training mini-batch number 234000\n",
      "Epoch 46: validation accuracy 97.67%\n",
      "Training mini-batch number 235000\n",
      "Training mini-batch number 236000\n",
      "Training mini-batch number 237000\n",
      "Training mini-batch number 238000\n",
      "Training mini-batch number 239000\n",
      "Epoch 47: validation accuracy 97.67%\n",
      "Training mini-batch number 240000\n",
      "Training mini-batch number 241000\n",
      "Training mini-batch number 242000\n",
      "Training mini-batch number 243000\n",
      "Training mini-batch number 244000\n",
      "Epoch 48: validation accuracy 97.67%\n",
      "Training mini-batch number 245000\n",
      "Training mini-batch number 246000\n",
      "Training mini-batch number 247000\n",
      "Training mini-batch number 248000\n",
      "Training mini-batch number 249000\n",
      "Epoch 49: validation accuracy 97.65%\n",
      "Training mini-batch number 250000\n",
      "Training mini-batch number 251000\n",
      "Training mini-batch number 252000\n",
      "Training mini-batch number 253000\n",
      "Training mini-batch number 254000\n",
      "Epoch 50: validation accuracy 97.66%\n",
      "Training mini-batch number 255000\n",
      "Training mini-batch number 256000\n",
      "Training mini-batch number 257000\n",
      "Training mini-batch number 258000\n",
      "Training mini-batch number 259000\n",
      "Epoch 51: validation accuracy 97.68%\n",
      "Training mini-batch number 260000\n",
      "Training mini-batch number 261000\n",
      "Training mini-batch number 262000\n",
      "Training mini-batch number 263000\n",
      "Training mini-batch number 264000\n",
      "Epoch 52: validation accuracy 97.67%\n",
      "Training mini-batch number 265000\n",
      "Training mini-batch number 266000\n",
      "Training mini-batch number 267000\n",
      "Training mini-batch number 268000\n",
      "Training mini-batch number 269000\n",
      "Epoch 53: validation accuracy 97.66%\n",
      "Training mini-batch number 270000\n",
      "Training mini-batch number 271000\n",
      "Training mini-batch number 272000\n",
      "Training mini-batch number 273000\n",
      "Training mini-batch number 274000\n",
      "Epoch 54: validation accuracy 97.66%\n",
      "Training mini-batch number 275000\n",
      "Training mini-batch number 276000\n",
      "Training mini-batch number 277000\n",
      "Training mini-batch number 278000\n",
      "Training mini-batch number 279000\n",
      "Epoch 55: validation accuracy 97.64%\n",
      "Training mini-batch number 280000\n",
      "Training mini-batch number 281000\n",
      "Training mini-batch number 282000\n",
      "Training mini-batch number 283000\n",
      "Training mini-batch number 284000\n",
      "Epoch 56: validation accuracy 97.63%\n",
      "Training mini-batch number 285000\n",
      "Training mini-batch number 286000\n",
      "Training mini-batch number 287000\n",
      "Training mini-batch number 288000\n",
      "Training mini-batch number 289000\n",
      "Epoch 57: validation accuracy 97.64%\n",
      "Training mini-batch number 290000\n",
      "Training mini-batch number 291000\n",
      "Training mini-batch number 292000\n",
      "Training mini-batch number 293000\n",
      "Training mini-batch number 294000\n",
      "Epoch 58: validation accuracy 97.64%\n",
      "Training mini-batch number 295000\n",
      "Training mini-batch number 296000\n",
      "Training mini-batch number 297000\n",
      "Training mini-batch number 298000\n",
      "Training mini-batch number 299000\n",
      "Epoch 59: validation accuracy 97.63%\n",
      "Finished training network.\n",
      "Best validation accuracy of 97.77% obtained at iteration 124999\n",
      "Corresponding test accuracy of 97.66%\n",
      "Training mini-batch number 0\n",
      "Training mini-batch number 1000\n",
      "Training mini-batch number 2000\n",
      "Training mini-batch number 3000\n",
      "Training mini-batch number 4000\n",
      "Epoch 0: validation accuracy 97.63%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.84%\n",
      "Training mini-batch number 5000\n",
      "Training mini-batch number 6000\n",
      "Training mini-batch number 7000\n",
      "Training mini-batch number 8000\n",
      "Training mini-batch number 9000\n",
      "Epoch 1: validation accuracy 97.63%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.84%\n",
      "Training mini-batch number 10000\n",
      "Training mini-batch number 11000\n",
      "Training mini-batch number 12000\n",
      "Training mini-batch number 13000\n",
      "Training mini-batch number 14000\n",
      "Epoch 2: validation accuracy 97.63%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.84%\n",
      "Training mini-batch number 15000\n",
      "Training mini-batch number 16000\n",
      "Training mini-batch number 17000\n",
      "Training mini-batch number 18000\n",
      "Training mini-batch number 19000\n",
      "Epoch 3: validation accuracy 97.64%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.84%\n",
      "Training mini-batch number 20000\n",
      "Training mini-batch number 21000\n",
      "Training mini-batch number 22000\n",
      "Training mini-batch number 23000\n",
      "Training mini-batch number 24000\n",
      "Epoch 4: validation accuracy 97.64%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.83%\n",
      "Training mini-batch number 25000\n",
      "Training mini-batch number 26000\n",
      "Training mini-batch number 27000\n",
      "Training mini-batch number 28000\n",
      "Training mini-batch number 29000\n",
      "Epoch 5: validation accuracy 97.65%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.81%\n",
      "Training mini-batch number 30000\n",
      "Training mini-batch number 31000\n",
      "Training mini-batch number 32000\n",
      "Training mini-batch number 33000\n",
      "Training mini-batch number 34000\n",
      "Epoch 6: validation accuracy 97.65%\n",
      "This is the best validation accuracy to date.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The corresponding test accuracy is 97.81%\n",
      "Training mini-batch number 35000\n",
      "Training mini-batch number 36000\n",
      "Training mini-batch number 37000\n",
      "Training mini-batch number 38000\n",
      "Training mini-batch number 39000\n",
      "Epoch 7: validation accuracy 97.66%\n",
      "This is the best validation accuracy to date.\n",
      "The corresponding test accuracy is 97.83%\n",
      "Training mini-batch number 40000\n",
      "Training mini-batch number 41000\n",
      "Training mini-batch number 42000\n",
      "Training mini-batch number 43000\n",
      "Training mini-batch number 44000\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "net.SGD(training_data, 60, mini_batch_size, 0.1, \n",
    "            validation_data, test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
